{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.special\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork:\n",
    "    # initialise the neural network\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate) -> None:\n",
    "        self.inodes = input_nodes\n",
    "        self.hnodes = hidden_nodes\n",
    "        self.onodes = output_nodes\n",
    "        self.lr = learning_rate\n",
    "\n",
    "        # set weight: input -> hidden, -0.5 ~ +0.5\n",
    "        # self.wih = np.random.rand(self.hnodes, self.inodes) - 0.5\n",
    "        self.wih = np.random.normal(0.0, pow(self.hnodes, -0.5), (self.hnodes, self.inodes))\n",
    "        # hidden -> output\n",
    "        # self.who = np.random.rand(self.onode, self.hnodes) - 0.5\n",
    "        self.who = np.random.normal(0.0, pow(self.onodes, -0.5), (self.onodes, self.hnodes))\n",
    "\n",
    "        # activation function: sigmoid = 1/(1+exp(-x))\n",
    "        self.activation_function = lambda x: scipy.special.expit(x)\n",
    "\n",
    "        # activation funciton for back query: x = ln(y/(1-y))\n",
    "        self.back_activation_func = lambda y: scipy.special.logit(y)\n",
    "\n",
    "    # train the neural network\n",
    "    def train(self, input_list, target_list):\n",
    "        inputs = np.array(input_list, ndmin=2).T\n",
    "        targets = np.array(target_list, ndmin=2).T\n",
    "        hidden_inputs = np.dot(self.wih, inputs)\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        final_inputs = np.dot(self.who, hidden_outputs)\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "\n",
    "        # calculate errors\n",
    "        output_errors = targets - final_outputs\n",
    "        hidden_errors = np.dot(self.who.T, output_errors)\n",
    "\n",
    "        # update weights\n",
    "        self.who += self.lr * np.dot(output_errors * final_outputs * (1.0 - final_outputs), np.transpose(hidden_outputs))\n",
    "        self.wih += self.lr * np.dot(hidden_errors * hidden_outputs * (1.0 - hidden_outputs), np.transpose(inputs))\n",
    "\n",
    "    # query the neural network\n",
    "    def query(self, input_list):\n",
    "        # convert input to 2d array\n",
    "        inputs = np.array(input_list, ndmin=2).T\n",
    "        hidden_inputs = np.dot(self.wih, inputs)\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        final_inputs = np.dot(self.who, hidden_outputs)\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "        return final_outputs\n",
    "\n",
    "    # back query, input label, output image\n",
    "    def back_query(self, label):\n",
    "        final_outputs = np.zeros(self.onodes) + 0.01\n",
    "        final_outputs[label] = 0.99\n",
    "\n",
    "        final_inputs = self.back_activation_func(final_outputs)\n",
    "\n",
    "        hidden_outputs = np.dot(self.who.T, final_inputs) # ?\n",
    "        # scale outputs to 0.01 ~ 0.99, as the sigmoid value region is (0, 1).\n",
    "        # and here we just handle all inputs values without excluding.\n",
    "        # so the scale is required to make sure all input for sigmoid is valid.\n",
    "        hidden_outputs -= np.min(hidden_outputs)\n",
    "        hidden_outputs /= np.max(hidden_outputs)\n",
    "        hidden_outputs *= 0.98\n",
    "        hidden_outputs += 0.01\n",
    "\n",
    "        hidden_inputs = self.back_activation_func(hidden_outputs)\n",
    "\n",
    "        inputs = np.dot(self.wih.T, hidden_inputs)\n",
    "        inputs -= np.min(inputs)\n",
    "        inputs /= np.max(inputs)\n",
    "        inputs *= 0.98\n",
    "        inputs += 0.01\n",
    "\n",
    "        return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data_file = open('mnist_dataset/mnist_train.csv', 'r')\n",
    "training_data_list = training_data_file.readlines()\n",
    "training_data_file.close()\n",
    "# test results\n",
    "test_data_file = open('mnist_dataset/mnist_test.csv', 'r')\n",
    "test_data_list = test_data_file.readlines()\n",
    "test_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def do_train_and_test(model, output_nodes, learning_rate, correct_rates, epochs):\n",
    "    for epoch in range(epochs):\n",
    "        for record in training_data_list:\n",
    "            all_values = record.split(',')\n",
    "            inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "            targets = np.zeros(output_nodes) + 0.01\n",
    "            targets[int(all_values[0])] = 0.99\n",
    "            model.train(inputs, targets)\n",
    "\n",
    "    # record scores\n",
    "    score_card = []\n",
    "    num_score = {0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0}\n",
    "    num_total = {0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0}\n",
    "    for record in test_data_list:\n",
    "        all_values = record.split(',')\n",
    "        correct_label = int(all_values[0])\n",
    "        num_total[correct_label] += 1\n",
    "        inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "        outputs = model.query(inputs)\n",
    "        # get index of max value \n",
    "        label = np.argmax(outputs)\n",
    "        if label == correct_label:\n",
    "            score_card.append(1)\n",
    "            num_score[label] += 1\n",
    "        else:\n",
    "            score_card.append(0)\n",
    "    score_card_array = np.asarray(score_card)\n",
    "    correct_rates[learning_rate] = score_card_array.sum() / score_card_array.size\n",
    "    for key in num_total:\n",
    "        if num_total[key] != 0:\n",
    "            num_score[key] /= num_total[key]\n",
    "    return num_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.9928571428571429,\n",
       " 1: 0.9876651982378855,\n",
       " 2: 0.9486434108527132,\n",
       " 3: 0.9544554455445544,\n",
       " 4: 0.960285132382892,\n",
       " 5: 0.9439461883408071,\n",
       " 6: 0.9707724425887265,\n",
       " 7: 0.9357976653696498,\n",
       " 8: 0.9527720739219713,\n",
       " 9: 0.956392467789891}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cost about 1 min 5 secs\n",
    "epochs = 2\n",
    "learning_rate = 0.2\n",
    "input_nodes = 784 # image is 28 x 28\n",
    "hidden_nodes = 100\n",
    "output_nodes = 10 # 0 ~ 9 total 10 labels\n",
    "correct_rates = {}\n",
    "n = NeuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)\n",
    "num_score = do_train_and_test(n, output_nodes, learning_rate, correct_rates, epochs)\n",
    "num_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.2: 0.9607}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recogonize customized numbers\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2572af19c70>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAOs0lEQVR4nO3df4gcdZrH8c9zcxFCDNFxxhiymuRUEDm9MQ7DwspGWRQNSLIqskEWD4JZQWGX5A9/BFxRgiKXXRY9lESjWbOXuP72D3PuGBfiIqwZ45xJlJxZTVzj6HSMGKOJuUye+2Mqy6hT357pru5q53m/YOjuerpSD4Ufq7u+Vf01dxeAie+fym4AQHMQdiAIwg4EQdiBIAg7EMQ/N3NjHR0dPnv27GZuEghl9+7d2rdvn41WqyvsZna5pN9JapP0sLvfm3r/7Nmz1dfXV88mASR0d3fn1mr+GG9mbZL+U9IVks6VtMjMzq313wPQWPV8Z++RtMvd33P3I5I2SFpQTFsAilZP2GdK+vuI1x9my77BzJaYWZ+Z9VUqlTo2B6AeDT8b7+6r3L3b3bs7OzsbvTkAOeoJ+15Jp494/YNsGYAWVE/Yt0g628zmmNkJkn4m6YVi2gJQtJqH3tz9qJndLOklDQ+9rXH3HYV1BqBQdY2zu/uLkl4sqBcADcTlskAQhB0IgrADQRB2IAjCDgRB2IEgmno/e1R796YvLLznnnuS9UcffTRZP/HEE3NrM2d+53aFbzjrrLOS9csuuyxZv+qqq5L19vb2ZB3Nw5EdCIKwA0EQdiAIwg4EQdiBIAg7EARDb01w3333JesPPvhgsn7s2LFk/auvvsqtDQ4OJtd98803k/WnnnoqWV+/fn2y/swzz+TWpk2bllwXxeLIDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM7eBOvWrUvWq42jl8ndk/VXXnklWb/yyitza88991xyXW6PLRZHdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgnH2JjjppJOS9f379zenkRK8+uqrubXHHnssue7SpUsL7ia2usJuZrslfSFpSNJRd+8uoikAxSviyH6Ju+8r4N8B0EB8ZweCqDfsLulPZvaGmS0Z7Q1mtsTM+sysr1Kp1Lk5ALWqN+wXuftcSVdIusnMfvztN7j7Knfvdvfuzs7OOjcHoFZ1hd3d92aPg5KeldRTRFMAildz2M1siplNPf5c0mWSthfVGIBi1XM2frqkZ83s+L/zX+7+34V0NcFU+934G264IVn/7LPPimynZTzwwAPJOuPsxao57O7+nqR/K7AXAA3E0BsQBGEHgiDsQBCEHQiCsANBcItrE8yfPz9Z37RpU7L+0ksvJes7duzIrW3cuDG57qeffpqsN9L777+frA8NDSXrbW1tRbYz4XFkB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgGGdvgsmTJyfrF1xwQbJ+/vnnJ+tffvllbq23tze57jXXXJOsl+nw4cPJ+pQpU5rUycTAkR0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgmCcvQCHDh1K1jdv3pys33333cn666+/nqwfPXo0t+buyXVb2ddff52sM84+PhzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtkL0N/fn6wvWrQoWZ+oUzLXa9asWcn6woULk/Ubb7wxtzZ37tzkutV+g+D7qOqR3czWmNmgmW0fsazdzHrN7N3s8eTGtgmgXmP5GP+YpMu/texWSZvc/WxJm7LXAFpY1bC7+2ZJ+7+1eIGktdnztZIWFtsWgKLVeoJuursPZM8/ljQ9741mtsTM+sysr1Kp1Lg5APWq+2y8D99pkXu3hbuvcvdud+/u7Oysd3MAalRr2D8xsxmSlD0OFtcSgEaoNewvSLo+e369pOeLaQdAo1QdZzez9ZIultRhZh9K+rWkeyX90cwWS9oj6dpGNtnqVq5cmawzjl6bgwcPJuvr1q1L1jds2JBbW758eXLdW265JVn/Po7DVw27u+ddEfKTgnsB0EBcLgsEQdiBIAg7EARhB4Ig7EAQ3OKKCSv1E9t33XVXct2Ojo5kffHixcl6Kw7NcWQHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZy/AsmXLkvUtW7Yk6x988EGR7WAMqk1lfdtttyXr5513XrI+b968cffUaBzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtkL0NXVlaw/+eSTyfrTTz+drO/cuTNZP3z4cG6t2n3VkyZNStY/+uijZH3Xrl3J+oEDB3Jrhw4dSq5bpmo/Y71ixYpknXF2AKUh7EAQhB0IgrADQRB2IAjCDgRB2IEgGGcvQLWx7J6enrrq32dDQ0O5tWr38Ve7p/yJJ56oqaci9Pb2lrbtWlU9spvZGjMbNLPtI5bdaWZ7zaw/+5vf2DYB1GssH+Mfk3T5KMt/6+5d2d+LxbYFoGhVw+7umyXtb0IvABqonhN0N5vZW9nH/JPz3mRmS8ysz8z6KpVKHZsDUI9aw/6gpDMldUkakLQy743uvsrdu929u7Ozs8bNAahXTWF390/cfcjdj0laLWnink4GJoiawm5mM0a8/Kmk7XnvBdAaqo6zm9l6SRdL6jCzDyX9WtLFZtYlySXtlvSLxrWI77O2trbc2pw5c5LrVptDvcxx9hNOOKG0bdeqatjdfdEoix9pQC8AGojLZYEgCDsQBGEHgiDsQBCEHQiCW1zRsrZt21Z2C7mOHDlSdgvjxpEdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnB2l+fzzz5P1pUuXNqmT8bv00kvLbmHcOLIDQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCMs6OhDh06lFu74447kutWm9K5TMuXLy+7hXHjyA4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOjrpUuyd9xYoVubX777+/6HYKU+1+9Z6eniZ1UpyqR3YzO93M/mxmb5vZDjP7Zba83cx6zezd7PHkxrcLoFZj+Rh/VNIydz9X0g8l3WRm50q6VdImdz9b0qbsNYAWVTXs7j7g7luz519IekfSTEkLJK3N3rZW0sIG9QigAOM6QWdmsyVdIOmvkqa7+0BW+ljS9Jx1lphZn5n1VSqVenoFUIcxh93MTpT0tKRfufuBkTV3d0k+2nruvsrdu929u7Ozs65mAdRuTGE3s0kaDvof3P2ZbPEnZjYjq8+QNNiYFgEUoerQm5mZpEckvePuvxlRekHS9ZLuzR6fb0iHqGpoaCi3tn///uS6W7duTdY3btyYrK9fvz5ZHxxszWNAe3t7sv7QQw8l65MnTy6ynaYYyzj7jyT9XNI2M+vPlt2u4ZD/0cwWS9oj6dqGdAigEFXD7u5/kWQ55Z8U2w6ARuFyWSAIwg4EQdiBIAg7EARhB4LgFtcxSv0kcrWx6tWrVyfrL7/8crJ+8ODBZD3V25EjR5LrTmTTpk3Lra1Zsya57qxZs4pup3Qc2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZx6i/vz+3dt111yXX3bNnT8HdQKp+T/rjjz+eW7vkkkuS67a1tdXUUyvjyA4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOPkYrV67MrTGO3hhnnnlmst7b25usn3HGGbm1iTiOXg1HdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IYizzs58u6feSpktySavc/XdmdqekGyRVsrfe7u4vNqpRTDxXX311sl5tjvSOjo4i25nwxnJRzVFJy9x9q5lNlfSGmR2/muG37v4fjWsPQFHGMj/7gKSB7PkXZvaOpJmNbgxAscb1nd3MZku6QNJfs0U3m9lbZrbGzE7OWWeJmfWZWV+lUhntLQCaYMxhN7MTJT0t6VfufkDSg5LOlNSl4SP/qBePu/sqd+929+7Ozs76OwZQkzGF3cwmaTjof3D3ZyTJ3T9x9yF3PyZptaSexrUJoF5Vw25mJukRSe+4+29GLJ8x4m0/lbS9+PYAFGUsZ+N/JOnnkraZWX+27HZJi8ysS8PDcbsl/aIB/bWMZcuW5dYGBgaS67722mtFt1OYqVOnJusXXnhhsj5v3rxkfcGCBbm1c845J7nu5MmTk3WMz1jOxv9Fko1SYkwd+B7hCjogCMIOBEHYgSAIOxAEYQeCIOxAEPyU9Bh1dXXl1h5++OHkujt37kzWh69bynfaaacl66eeempu7ZRTTkmuO2XKlGQ94k8uT1Qc2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCHP35m3MrCJp5PzGHZL2Na2B8WnV3lq1L4nealVkb7PcfdTff2tq2L+zcbM+d+8urYGEVu2tVfuS6K1WzeqNj/FAEIQdCKLssK8qefsprdpbq/Yl0VutmtJbqd/ZATRP2Ud2AE1C2IEgSgm7mV1uZjvNbJeZ3VpGD3nMbLeZbTOzfjPrK7mXNWY2aGbbRyxrN7NeM3s3exx1jr2ServTzPZm+67fzOaX1NvpZvZnM3vbzHaY2S+z5aXuu0RfTdlvTf/ObmZtkv5X0qWSPpS0RdIid3+7qY3kMLPdkrrdvfQLMMzsx5IOSvq9u/9rtuw+Sfvd/d7sf5Qnu/stLdLbnZIOlj2NdzZb0YyR04xLWijp31Xivkv0da2asN/KOLL3SNrl7u+5+xFJGyTlTxsSmLtvlrT/W4sXSFqbPV+r4f9Ymi6nt5bg7gPuvjV7/oWk49OMl7rvEn01RRlhnynp7yNef6jWmu/dJf3JzN4wsyVlNzOK6e5+fL6pjyVNL7OZUVSdxruZvjXNeMvsu1qmP68XJ+i+6yJ3nyvpCkk3ZR9XW5IPfwdrpbHTMU3j3SyjTDP+D2Xuu1qnP69XGWHfK+n0Ea9/kC1rCe6+N3sclPSsWm8q6k+Oz6CbPQ6W3M8/tNI03qNNM64W2HdlTn9eRti3SDrbzOaY2QmSfibphRL6+A4zm5KdOJGZTZF0mVpvKuoXJF2fPb9e0vMl9vINrTKNd9404yp535U+/bm7N/1P0nwNn5H/m6TlZfSQ09e/SPqf7G9H2b1JWq/hj3X/p+FzG4slnSJpk6R3Jb0sqb2Fentc0jZJb2k4WDNK6u0iDX9Ef0tSf/Y3v+x9l+irKfuNy2WBIDhBBwRB2IEgCDsQBGEHgiDsQBCEHQiCsANB/D/UUXEigcyDdwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = cv2.imread('images/3-2.png', cv2.IMREAD_GRAYSCALE)\n",
    "img_data = (255.0 - img.reshape(28, 28)) / 255.0 * 0.99 + 0.01\n",
    "predicts = n.query(img_data.flatten())\n",
    "print(np.argmax(predicts))\n",
    "plt.imshow(img_data, cmap='Greys', interpolation='None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2572b283220>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVtElEQVR4nO3dW2yd1ZUH8P+KQyDEJM7VFzA4MeESAwlgRUDRKCM0hYYH0hdUHipGQqQPRWqlPgxiHsojGk1b9WFUyR2g6ahDVakgeEAzpYhbJVTFuUASO5ALTurEl9ycxDEJsb3mwV86Bvz9l3u+43OOuv8/KbJzVvY5+1xWjn3WXnubu0NE/v7Nq/YERKQylOwiiVCyiyRCyS6SCCW7SCLmV/LGGhoavKmpKTceVQYuX76cG5s/n98VM6Px6LbZ+Oi6x8fHaTyaexHR/Yri8+bx94Povk1OTubGrrrqqkK3za4bACYmJnJj0f2uq6uj8aKPK3vNRLfNDA4OYmRkZMYrL/QqM7NHAPwcQB2A/3T3F9i/b2pqQldXV248euEMDg7mxpYtW0bHLly4kMY///xzGr/mmmtyYwsWLKBjh4eHaTyae4S9OKLH9OLFizR+3XXX0fjJkydpfHR0NDfG/uOfzW2fP3++5Dh74wCAhoYGGo8et+hxZ89ZdNvM008/nRsr+cd4M6sD8B8AvgVgHYAnzGxdqdcnInOryO/sGwEcdPfD7v4FgN8CeKw80xKRciuS7NcD+Mu0v/dnl32JmW01s24z6x4ZGSlwcyJSxJx/Gu/uXe7e6e6dRX4XEZFiiiT7MQCt0/5+Q3aZiNSgIsm+HcBaM1ttZgsAfAfAG+WZloiUW8mlN3cfN7NnAPwvpkpvL7n7PjZmYmKClmKiuuny5cvZfOjYqBYelWLYvK+++upCtx3d76gsyK6fzRuIa92sVg3E5bGzZ8/mxr744gs69ujRozQelVOZ+vp6Go9eD1HpbenSpTR+7bXX5sYuXLhAx7I4K/kVqrO7+5sA3ixyHSJSGVouK5IIJbtIIpTsIolQsoskQskukgglu0giKtrPbma0rhv18bL6ImtBBYATJ07QeFR3ja6fiWrZUb2Y1WQBYGhoKDcW1cGL9m1Hc2NtrFE9uaWlhcajx40959FzEon2IIjWEJw+fTo3Fr3W2BoA9nzpnV0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRFS09DY+Pk53Wo12WV20aFFuLCoRLV68mMajbYtZiSq67ahN9NSpUzQelYnYrrtRq2U092jn3GiH1zNnzuTGohJTf38/jTc2NtL4ihUrcmNRaSya2/79+2k8es7Yazlqr2VlP9burHd2kUQo2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJREXr7HV1dbSWHm3JzOqLUbtkVDctcnRxNO+ojh61avb29tL4pUuXcmPRKavHjx+n8SVLltA4q2UDvHX43LlzdGy0XXP0nLPb7ujooGOjluiodThat8Ge84GBATqWrZ1gt6t3dpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSoWQXSURF6+wAP1I26vtmoq19o/7i6NjkqG7KNDQ00Hh0JHO0XTOrR0fX/cknn9B4VE+O7tuaNWtyY+w4ZyCe2y233ELjbE3H4cOH6dg77riDxqP7He0D0NfXlxuL1oSwPQRYDhVKdjPrA3AewASAcXfvLHJ9IjJ3yvHO/o/uzpdpiUjV6Xd2kUQUTXYH8Acz22FmW2f6B2a21cy6zaw7+h1NROZO0R/jH3T3Y2a2CsBbZrbf3d+f/g/cvQtAFwCsXbuWd5uIyJwp9M7u7seyr8MAXgOwsRyTEpHyKznZzWyRmV135XsA3wSwt1wTE5HyKvJjfCOA17J9qucD+G93/x82YN68eXT/9qNHj9IbZH280dHDUe2S7W8O8Dp8VOOPavjR3KL7xurw0f7oUb/72NgYjUf3jdXp2d7pALBq1Soa//TTT2n8tttuy43t3LmTjo16yrds2ULj0dzb2tpyY2x/AoAf98zWg5Sc7O5+GMD6UseLSGWp9CaSCCW7SCKU7CKJULKLJELJLpKIire4Mi0tLTTOWvuiMg47KhqIS0isRNXc3EzHRttUv/feezQezX3lypW5saj0tnr1ahqPxkfbPbO5R+Wtd999l8ajNlLW1tzU1ETHRku7e3p6aDx6XNjjGr1e2NblOrJZRJTsIqlQsoskQskukgglu0gilOwiiVCyiySionV2d6f1xcuXL9PxrJYeHf8bbYnMavgAr19G2zVH2xZHxwOzLZGj+Pbt2+nYaJvqV199lcajWjerGd933310bGNjI41HtWxWC2ctpgDfAhuIX2/R1uXsqOuo7Zi1VKvOLiJKdpFUKNlFEqFkF0mEkl0kEUp2kUQo2UUSUdE6++XLl2kPM6s9AsCFCxdyY1E/e1TLjuqi7Kjp6DhnNm8AqK+vp3HWrw7w/uY777yTjv3www9pPOp3v/fee2mc9fofOXKEju3o6KDx6DkdHR3NjbW3t9Ox0Wuxt7eXxu+66y4aZ9uiR+tNWB2e7cugd3aRRCjZRRKhZBdJhJJdJBFKdpFEKNlFEqFkF0lERevsdXV1tKYc9ZwfO3YsN7Zw4UI6NjpCN+rLZscqR2OjnvGodzrqZ2ePS9Qz3tDQQONLliyh8c7OThpndfaXX36Zjo362aNaNut3j/aNj57TaO1DtAagtbU1Nxa9lksVvrOb2UtmNmxme6ddtszM3jKzA9nX/BUCIlITZvNj/K8APPKVy54F8La7rwXwdvZ3EalhYbK7+/sATn/l4scAbMu+3wZgS3mnJSLlVuoHdI3ufmWR+yCA3F+uzGyrmXWbWXd0fpaIzJ3Cn8b71I6CubsKunuXu3e6e2f0YY+IzJ1Sk33IzJoBIPvKjxkVkaorNdnfAPBk9v2TAF4vz3REZK6EdXYzewXAJgArzKwfwI8BvADgd2b2FIAjAB4vx2RYvRjgtfKoZ5zVNYG4H/706a9+Rvn/on3jo19fovjY2BiNs3p0dHY869MH4r7ujRs3lnz9hw4domOjWviZM2donJ1RMDExQceyPQKAeO0D278dAI4ePZobi2r8pe4bHya7uz+RE3ooGisitUPLZUUSoWQXSYSSXSQRSnaRRCjZRRJR8RZXVrKIyiGs1BK1akZLdaPjf1m5Y/369XRsX18fjRct3bG5DQ4O0rFRmef++++n8ah1+NKlS7mxzZs307EHDx6k8agkGbVMMx988AGNnzp1isaHh/k6s4cffjg3FpWR2WtdRzaLiJJdJBVKdpFEKNlFEqFkF0mEkl0kEUp2kURUvM6+ePHi3DhrIwX4FrtRbTLannf//v00zo7R3bdvHx0bibaajtYAsBbX6H6PjIzQeFSHj9ZGsKOw2fHCQDz36JhttkV3VCc/d+4cjbO1DQDQ0tJC4+z1GrXHRo95Hr2ziyRCyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIipaZx8fH6d9vlGtnPXCsyOVAeCzzz6j8aGhIRofHR3NjR0/fpyOjfqqoy2Ro+2cd+3alRt79NFH6dh58/j/9/39/TQe1ZOXLs0/4LdorZutfQCAw4cP58aiPQTWrVtH49Hcoq2o2esp2tacbZE9dUDTzPTOLpIIJbtIIpTsIolQsoskQskukgglu0gilOwiiahonR3gfcBR/zKrIUZHD0c13Whv97vvvjs31tvbS8dGdfioLzvqtb/nnntyY+fPn6djly9fTuOsTg7E+/Gz/Qui9QesTg4Ue85Xr15d6LqjI5ujdR/19fW5sej1wPY/YOsmwnd2M3vJzIbNbO+0y543s2Nmtjv7w3f7F5Gqm82P8b8C8MgMl//M3Tdkf94s77REpNzCZHf39wHw/aJEpOYV+YDuGTP7OPsxP/cXOzPbambdZtYd/X4nInOn1GT/BYB2ABsADAD4Sd4/dPcud+90987ogEIRmTslJbu7D7n7hLtPAvglgI3lnZaIlFtJyW5mzdP++m0Ae/P+rYjUhrDObmavANgEYIWZ9QP4MYBNZrYBgAPoA/C92d4gq19G+2GzHuG6ujo6lvUPA8CJEydo/OTJk7mxor3yHR0dNB7tK79hw4bc2I033kjHRr3T0fnuUT2ZPafR/gVF9jcAeC9+VEeP1h9Ec4v2CWC9+GxtAsDPvGd78YfJ7u5PzHDxi9E4EaktWi4rkgglu0gilOwiiVCyiyRCyS6SiIq3uDJjY2M0zlr/otV5N910E42z7ZgBXrqLSoZRK2fUhhq1/h44cCA3tnbtWjo22lI5ek6iubPHlbUsA0B7ezuN79ixg8ZZG2nk0KFDNB61yLJjtAHefhs95qWWr/XOLpIIJbtIIpTsIolQsoskQskukgglu0gilOwiiahonX1ycpIeNxu1Y7JjchcsWEDHRjXdqJbNtoOO2hmj9tuo1h2157L1B9GxxtEagI8++ojG9+zZQ+Os1r1q1So6Ntr+O2pxZa+XqKU5aoGNntPoGG62LiTaSpq1uBbaSlpE/j4o2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJREXr7PPnz0dDQ0NuPDqit6WlJTfGao9XbpuJaps9PT25sba2NjqWbWkMxL3R0bbFR44cyY298847dGw09+g4araNNcDXCES17GhL5ZGRERpnLl68SOPR66nIFtpAvD04w2r8qrOLiJJdJBVKdpFEKNlFEqFkF0mEkl0kEUp2kURUtM5uZvTYZVZHvzI+T9RTHvUXR3XT22+/PTcWHWsc1dGj8VFNlvVmRzX6vr4+Gt+0aRONR732rJ89ek6iWnV03DTbJyCq8d96662Fbpsd8Q0AZ8+eLfm62bHMbN+G8J3dzFrN7B0z6zGzfWb2g+zyZWb2lpkdyL4uja5LRKpnNj/GjwP4kbuvA3AfgO+b2ToAzwJ4293XAng7+7uI1Kgw2d19wN13Zt+fB9AL4HoAjwHYlv2zbQC2zNEcRaQM/qYP6MysDcDdAP4MoNHdB7LQIIAZD7cys61m1m1m3UXWMotIMbNOdjOrB/B7AD909y/t5OdTnwrM+MmAu3e5e6e7d7ImGBGZW7NKdjO7ClOJ/ht3fzW7eMjMmrN4M4DhuZmiiJRDWHqzqXrXiwB63f2n00JvAHgSwAvZ19ej6xofH6dlomi7Z3Y8MCvpAXE5Y/369TTOSinRds1Rq2ZUcoxKVGxua9asoWNvuOEGGo9aQaOS5f79+3Nj0bHGy5cvp3G2vTfAS55RqTa67agsGG01zcrIQ0NDdCxr12bzmk2d/RsAvgtgj5ntzi57DlNJ/jszewrAEQCPz+K6RKRKwmR39z8ByPtv6KHyTkdE5oqWy4okQskukgglu0gilOwiiVCyiySioi2uAK+lR9s5s3p0dGRz1MoZHR/MWhKj456jlYOs5grEbaqtra0lX/fY2BiNd3d303hkeDh/rVVUJ3/ggQdoPLpvbO0Fa70F4q2ko8ctWp/A5saONQeAlStX5sZYDV7v7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukoiK1tnr6uqwZMmS3HjUt3369OncWHNzMx0b9W2z6wZ4/zO7T0B8XHQ096gmvGvXrtxY1G8e9W1Hx2hH6xNYPbmpqYmOje53tEU3W3vBjrkG+LoKAFi2bBmNr1ixgsZZHb+jo4OOLXV7N72ziyRCyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIipaZ5+cnKTH6EZ7u7NjdqMe4Ggf72gfcFZXjfaFj45cjvqy29raaHzjxo25sZ6eHjo26rt+6KFiGwifO3cuN3bzzTfTsVGdPdqvf2BgIDcWvV6ixzxalxEdw83WH0T3ix3ZzGJ6ZxdJhJJdJBFKdpFEKNlFEqFkF0mEkl0kEUp2kUTM5nz2VgC/BtAIwAF0ufvPzex5AE8DuHLg+nPu/ia7LnendUB2/jrAe6ej2mRUy4560tle3ew+zSYenUs/OjpK46yWHdWqo/sdzY2tmwD44xaNZesqgLhWztY3FOk3j657NthzFp1bf/LkydwYe63NZlHNOIAfuftOM7sOwA4zeyuL/czd/30W1yEiVTab89kHAAxk3583s14A18/1xESkvP6m39nNrA3A3QD+nF30jJl9bGYvmdnSnDFbzazbzLrZjy4iMrdmnexmVg/g9wB+6O7nAPwCQDuADZh65//JTOPcvcvdO929M1pDLiJzZ1bJbmZXYSrRf+PurwKAuw+5+4S7TwL4JYD8bgwRqbow2W3qY+wXAfS6+0+nXT59S9RvA9hb/umJSLnM5tP4bwD4LoA9ZrY7u+w5AE+Y2QZMleP6AHwvuqK6ujraxhod2czKIVEZJyp/Rb9isJbGaCw7thgAli6d8eOOv4pKb2xr4ag8xY7BBuItlaPrZ89ZdNR11PIcPe6sbTkq87IWVKB4qZcd4x21HUfXnWc2n8b/CcBM105r6iJSW7SCTiQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEVHQr6YmJiULtmKw2yo5UBuKabpE6fFRzjY4mjo7gjY4HZnOLtsiOjnRmLaoAsG/fPhpvb2/PjRWtVV+4cIHG2fqFaCvo6CjraF1HVKenragFjvhma1X0zi6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIomwaKvgst6Y2QkAR6ZdtAJA/r641VWrc6vVeQGaW6nKObeb3H3GxREVTfav3bhZt7t3Vm0CRK3OrVbnBWhuparU3PRjvEgilOwiiah2sndV+faZWp1brc4L0NxKVZG5VfV3dhGpnGq/s4tIhSjZRRJRlWQ3s0fM7BMzO2hmz1ZjDnnMrM/M9pjZbjPrrvJcXjKzYTPbO+2yZWb2lpkdyL7yTecrO7fnzexY9tjtNrPNVZpbq5m9Y2Y9ZrbPzH6QXV7Vx47MqyKPW8V/ZzezOgCfAvgnAP0AtgN4wt17KjqRHGbWB6DT3au+AMPM/gHAKIBfu/sd2WX/BuC0u7+Q/Ue51N3/pUbm9jyA0Wof452dVtQ8/ZhxAFsA/DOq+NiReT2OCjxu1Xhn3wjgoLsfdvcvAPwWwGNVmEfNc/f3AXx1S5XHAGzLvt+GqRdLxeXMrSa4+4C778y+Pw/gyjHjVX3syLwqohrJfj2Av0z7ez9q67x3B/AHM9thZlurPZkZNLr7QPb9IIDGak5mBuEx3pX0lWPGa+axK+X486L0Ad3XPeju9wD4FoDvZz+u1iSf+h2slmqnszrGu1JmOGb8r6r52JV6/HlR1Uj2YwBap/39huyymuDux7KvwwBeQ+0dRT105QTd7Cs/NbKCaukY75mOGUcNPHbVPP68Gsm+HcBaM1ttZgsAfAfAG1WYx9eY2aLsgxOY2SIA30TtHUX9BoAns++fBPB6FefyJbVyjHfeMeOo8mNX9ePP3b3ifwBsxtQn8ocA/Gs15pAzrzUAPsr+7Kv23AC8gqkf6y5j6rONpwAsB/A2gAMA/ghgWQ3N7b8A7AHwMaYSq7lKc3sQUz+ifwxgd/Znc7UfOzKvijxuWi4rkgh9QCeSCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIon4PyobFYiEzbWZAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate image by input label\n",
    "img_data = n.back_query(0)\n",
    "plt.imshow(img_data.reshape(28, 28), cmap='Greys', interpolation='None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-9.866445447717885, 42.62776777226142)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_outputs = np.zeros(n.onodes) + 0.01\n",
    "final_outputs[0] = 0.99\n",
    "\n",
    "final_inputs = n.back_activation_func(final_outputs)\n",
    "\n",
    "hidden_outputs = np.dot(n.who.T, final_inputs) # \n",
    "np.min(hidden_outputs), np.max(hidden_outputs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 1.0)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# scale outputs to 0.01 ~ 0.99, as the sigmoid value region is (0, 1).\n",
    "# and here we just handle all inputs values without excluding.\n",
    "# so the scale is required to make sure all input for sigmoid is valid.\n",
    "hidden_outputs -= np.min(hidden_outputs)\n",
    "hidden_outputs /= np.max(hidden_outputs)\n",
    "\n",
    "np.min(hidden_outputs), np.max(hidden_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.01, 0.99)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden_outputs *= 0.98\n",
    "hidden_outputs += 0.01\n",
    "np.min(hidden_outputs), np.max(hidden_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('ai')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0cb0046d74bfb5a9fee6ebb85b9850d762e40d6b321d5d9a0aa0313eefa3c5b4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
